{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "tabnet_final.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bPFV2FX0FHKS",
        "colab_type": "text"
      },
      "source": [
        "# **Install / Import / Read Data :**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TVYdQrLD4FLC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 222
        },
        "outputId": "949608f9-d2e1-4003-bb36-0ef2a64c0aed"
      },
      "source": [
        "!pip install pytorch-tabnet"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pytorch-tabnet\n",
            "  Downloading https://files.pythonhosted.org/packages/6e/ee/1670a8072b03a42b16db79e45f7705c3f155c35f9ae83610b46c6d07a4e4/pytorch_tabnet-1.2.0-py3-none-any.whl\n",
            "Requirement already satisfied: scipy>1.4 in /usr/local/lib/python3.6/dist-packages (from pytorch-tabnet) (1.4.1)\n",
            "Requirement already satisfied: tqdm<5.0,>=4.36 in /usr/local/lib/python3.6/dist-packages (from pytorch-tabnet) (4.41.1)\n",
            "Requirement already satisfied: scikit_learn>0.21 in /usr/local/lib/python3.6/dist-packages (from pytorch-tabnet) (0.22.2.post1)\n",
            "Requirement already satisfied: torch<2.0,>=1.2 in /usr/local/lib/python3.6/dist-packages (from pytorch-tabnet) (1.6.0+cu101)\n",
            "Requirement already satisfied: numpy<2.0,>=1.17 in /usr/local/lib/python3.6/dist-packages (from pytorch-tabnet) (1.18.5)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit_learn>0.21->pytorch-tabnet) (0.16.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch<2.0,>=1.2->pytorch-tabnet) (0.16.0)\n",
            "Installing collected packages: pytorch-tabnet\n",
            "Successfully installed pytorch-tabnet-1.2.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mhugNzECO7-9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "fbe848fc-3606-4ff8-e76c-efc967dc953a"
      },
      "source": [
        "import pandas as pd \n",
        "import numpy as np\n",
        "from google.colab import drive\n",
        "pd.options.mode.chained_assignment = None\n",
        "pd.set_option('display.max_columns', None)\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.metrics import *\n",
        "from sklearn.model_selection import *\n",
        "drive.mount(\"/content/drive\")\n",
        "path = \"/content/drive/My Drive/MachineHack/Anomalies_Detection/\"\n",
        "\n",
        "\n",
        "train_data                          = pd.read_csv(path+\"Train.csv\")\n",
        "test_data                           = pd.read_csv(path+\"Test.csv\")\n",
        "train_y                             = train_data.Class.values\n",
        "train_data                          = train_data.drop(['Class'],axis=1)    \n"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OEEPejKeE0LT",
        "colab_type": "text"
      },
      "source": [
        "Eliminating features with low sum of ones :"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y68nKJZhO8Bo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "625789e0-99bf-4a56-b341-811d0be1dc26"
      },
      "source": [
        "columns         = train_data.columns\n",
        "num_cols        = ['feature_1','feature_2','feature_3']\n",
        "cat_cols        = list(set(columns)-set(num_cols))\n",
        "needed_cat_cols  = []\n",
        "for cols in cat_cols:\n",
        "    if (train_data[cols].sum()>0):\n",
        "        needed_cat_cols.append(cols)\n",
        "        \n",
        "train_df         = train_data[needed_cat_cols+num_cols]\n",
        "test_df          = test_data[needed_cat_cols+num_cols]\n",
        "concat_df        = pd.concat((train_df,test_df),axis=0)\n",
        "print(train_df.shape,test_df.shape)\n",
        "\n",
        "\n"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(1763, 1522) (756, 1522)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rkNf8t9dO8Ed",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train            = train_df.values\n",
        "test             = test_df.values"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fucEogSBE57x",
        "colab_type": "text"
      },
      "source": [
        "# **Modeling :**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GJzQL2p3EU2o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from pytorch_tabnet.tab_model import TabNetClassifier\n",
        "import torch"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YcXC2gsHOxnv",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "92fa8f51-bc39-48dc-f20f-35885a5bc9cd"
      },
      "source": [
        "oof_pred               = np.zeros((len(train),))\n",
        "y_pred_final           = np.zeros((len(test), ))\n",
        "num_models             = 1\n",
        "\n",
        "n_splits               = 15\n",
        "error                  = []\n",
        "kf                     = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=42)\n",
        "\n",
        "for fold, (tr_ind, val_ind) in enumerate(kf.split(train, train_y)):\n",
        "    \n",
        "    \n",
        "    \n",
        "    X_train, X_val     = train[tr_ind], train[val_ind]\n",
        "    y_train, y_val     = train_y[tr_ind], train_y[val_ind]\n",
        "    \n",
        "\n",
        "\n",
        "    clf = TabNetClassifier(\n",
        "                            n_d=64, n_a=64, n_steps=5,\n",
        "                            gamma=1.5, n_independent=2, n_shared=2,\n",
        "                            \n",
        "                            lambda_sparse=1e-4, momentum=0.3, clip_value=2.,\n",
        "                            optimizer_fn=torch.optim.AdamW,\n",
        "                            optimizer_params=dict(lr=5e-2),\n",
        "                            scheduler_params = {\"gamma\": 0.95,\n",
        "                                            \"step_size\": 20},\n",
        "                            scheduler_fn=torch.optim.lr_scheduler.StepLR, epsilon=1e-15,\n",
        "                           seed = 1997\n",
        "                        )\n",
        "\n",
        "    \n",
        "    np.random.seed(seed=42)\n",
        "    clf.fit(\n",
        "                X_train=X_train, y_train=y_train,\n",
        "                X_valid=X_val, y_valid=y_val,\n",
        "                max_epochs=25, patience=100,\n",
        "                batch_size=32, virtual_batch_size=16,\n",
        "                \n",
        "            )\n",
        "\n",
        "    val_pred1          = clf.predict_proba(X_val)[:,1]\n",
        "#     auc_roc.append(roc_auc_score(y_val,val_pred1))\n",
        "    print('validation roc_auc model 1 fold-',fold+1,': ',roc_auc_score(y_val,val_pred1))\n",
        "    \n",
        "    \n",
        "    val_pred           = val_pred1\n",
        "    #print('validation ROC_AUC fold-',fold+1,': ',roc_auc_score(y_val, val_pred))\n",
        "    \n",
        "    oof_pred[val_ind]  = val_pred\n",
        "    \n",
        "    y_pred_final += (clf.predict_proba(test)[:,1])/(n_splits)\n",
        "    \n",
        "#     print('\\n')\n",
        "    \n",
        "print('OOF ROC_AUC:- ',(roc_auc_score(train_y,oof_pred)))"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Device used : cuda\n",
            "Will train until validation stopping metric hasn't improved in 100 rounds.\n",
            "---------------------------------------\n",
            "| EPOCH |  train  |   valid  | total time (s)\n",
            "| 1     | 0.50134 |  0.52500 |   2.7       \n",
            "| 2     | 0.61021 |  0.75602 |   5.2       \n",
            "| 3     | 0.62456 |  0.70278 |   7.4       \n",
            "| 4     | 0.65965 |  0.66343 |   9.6       \n",
            "| 5     | 0.68952 |  0.62546 |   11.9      \n",
            "| 6     | 0.69841 |  0.85602 |   14.1      \n",
            "| 7     | 0.76796 |  0.79676 |   16.4      \n",
            "| 8     | 0.72611 |  0.75880 |   18.8      \n",
            "| 9     | 0.80771 |  0.71528 |   21.1      \n",
            "| 10    | 0.84913 |  0.86667 |   23.6      \n",
            "| 11    | 0.86328 |  0.79676 |   25.8      \n",
            "| 12    | 0.85541 |  0.88935 |   28.1      \n",
            "| 13    | 0.88129 |  0.79583 |   30.4      \n",
            "| 14    | 0.84872 |  0.90463 |   32.7      \n",
            "| 15    | 0.87974 |  0.93241 |   34.9      \n",
            "| 16    | 0.90699 |  0.87731 |   37.0      \n",
            "| 17    | 0.90822 |  0.87361 |   39.1      \n",
            "| 18    | 0.91235 |  0.94491 |   41.3      \n",
            "| 19    | 0.90763 |  0.85370 |   43.5      \n",
            "| 20    | 0.91900 |  0.92037 |   45.7      \n",
            "| 21    | 0.90830 |  0.91343 |   47.9      \n",
            "| 22    | 0.91407 |  0.77639 |   50.1      \n",
            "| 23    | 0.85559 |  0.79028 |   52.2      \n",
            "| 24    | 0.88039 |  0.90694 |   54.3      \n",
            "| 25    | 0.89441 |  0.91759 |   56.5      \n",
            "Training done in 56.461 seconds.\n",
            "---------------------------------------\n",
            "validation roc_auc model 1 fold- 1 :  0.9449074074074074\n",
            "Device used : cuda\n",
            "Will train until validation stopping metric hasn't improved in 100 rounds.\n",
            "---------------------------------------\n",
            "| EPOCH |  train  |   valid  | total time (s)\n",
            "| 1     | 0.45566 |  0.33889 |   2.3       \n",
            "| 2     | 0.52375 |  0.51759 |   4.7       \n",
            "| 3     | 0.66347 |  0.65787 |   7.0       \n",
            "| 4     | 0.60403 |  0.68472 |   9.3       \n",
            "| 5     | 0.73717 |  0.69167 |   11.6      \n",
            "| 6     | 0.72724 |  0.70278 |   13.7      \n",
            "| 7     | 0.75738 |  0.65324 |   16.0      \n",
            "| 8     | 0.80323 |  0.65741 |   18.3      \n",
            "| 9     | 0.80027 |  0.65509 |   20.5      \n",
            "| 10    | 0.84875 |  0.72778 |   22.8      \n",
            "| 11    | 0.85754 |  0.66620 |   25.2      \n",
            "| 12    | 0.85816 |  0.68657 |   27.4      \n",
            "| 13    | 0.88593 |  0.70556 |   29.7      \n",
            "| 14    | 0.91457 |  0.70648 |   32.1      \n",
            "| 15    | 0.92435 |  0.71852 |   34.5      \n",
            "| 16    | 0.92941 |  0.68472 |   36.9      \n",
            "| 17    | 0.93928 |  0.75509 |   39.2      \n",
            "| 18    | 0.92998 |  0.74444 |   41.5      \n",
            "| 19    | 0.94632 |  0.73565 |   43.8      \n",
            "| 20    | 0.93051 |  0.75139 |   46.0      \n",
            "| 21    | 0.94177 |  0.78287 |   48.3      \n",
            "| 22    | 0.93353 |  0.74306 |   50.6      \n",
            "| 23    | 0.94412 |  0.70972 |   52.8      \n",
            "| 24    | 0.93923 |  0.75648 |   55.1      \n",
            "| 25    | 0.93130 |  0.75231 |   57.5      \n",
            "Training done in 57.544 seconds.\n",
            "---------------------------------------\n",
            "validation roc_auc model 1 fold- 2 :  0.7828703703703703\n",
            "Device used : cuda\n",
            "Will train until validation stopping metric hasn't improved in 100 rounds.\n",
            "---------------------------------------\n",
            "| EPOCH |  train  |   valid  | total time (s)\n",
            "| 1     | 0.55341 |  0.67315 |   2.4       \n",
            "| 2     | 0.60173 |  0.89676 |   4.7       \n",
            "| 3     | 0.72452 |  0.77824 |   7.0       \n",
            "| 4     | 0.61779 |  0.76806 |   9.2       \n",
            "| 5     | 0.71589 |  0.88009 |   11.5      \n",
            "| 6     | 0.78988 |  0.89907 |   13.8      \n",
            "| 7     | 0.79205 |  0.84630 |   16.0      \n",
            "| 8     | 0.79821 |  0.83889 |   18.2      \n",
            "| 9     | 0.81864 |  0.88056 |   20.7      \n",
            "| 10    | 0.81181 |  0.92130 |   23.0      \n",
            "| 11    | 0.77250 |  0.81528 |   25.2      \n",
            "| 12    | 0.78369 |  0.83935 |   27.4      \n",
            "| 13    | 0.81647 |  0.82731 |   29.7      \n",
            "| 14    | 0.78039 |  0.82222 |   31.9      \n",
            "| 15    | 0.79265 |  0.87083 |   34.3      \n",
            "| 16    | 0.84093 |  0.86667 |   36.5      \n",
            "| 17    | 0.81492 |  0.85463 |   38.8      \n",
            "| 18    | 0.78458 |  0.79537 |   41.0      \n",
            "| 19    | 0.81063 |  0.86343 |   43.4      \n",
            "| 20    | 0.78338 |  0.80602 |   45.7      \n",
            "| 21    | 0.81095 |  0.87593 |   48.2      \n",
            "| 22    | 0.84127 |  0.85046 |   50.4      \n",
            "| 23    | 0.82958 |  0.73009 |   52.7      \n",
            "| 24    | 0.84590 |  0.89907 |   55.0      \n",
            "| 25    | 0.79111 |  0.77222 |   57.3      \n",
            "Training done in 57.349 seconds.\n",
            "---------------------------------------\n",
            "validation roc_auc model 1 fold- 3 :  0.9212962962962962\n",
            "Device used : cuda\n",
            "Will train until validation stopping metric hasn't improved in 100 rounds.\n",
            "---------------------------------------\n",
            "| EPOCH |  train  |   valid  | total time (s)\n",
            "| 1     | 0.53513 |  0.63333 |   2.4       \n",
            "| 2     | 0.58228 |  0.61250 |   4.7       \n",
            "| 3     | 0.57820 |  0.63241 |   7.3       \n",
            "| 4     | 0.65428 |  0.69861 |   9.5       \n",
            "| 5     | 0.63561 |  0.80787 |   11.7      \n",
            "| 6     | 0.71817 |  0.78519 |   14.1      \n",
            "| 7     | 0.72446 |  0.73380 |   16.5      \n",
            "| 8     | 0.63641 |  0.76944 |   18.8      \n",
            "| 9     | 0.73891 |  0.75185 |   21.2      \n",
            "| 10    | 0.78275 |  0.71389 |   23.5      \n",
            "| 11    | 0.80772 |  0.73380 |   25.8      \n",
            "| 12    | 0.79475 |  0.79167 |   28.3      \n",
            "| 13    | 0.82605 |  0.63935 |   30.6      \n",
            "| 14    | 0.82512 |  0.82454 |   33.0      \n",
            "| 15    | 0.87281 |  0.87222 |   35.4      \n",
            "| 16    | 0.89334 |  0.84120 |   37.9      \n",
            "| 17    | 0.87788 |  0.82130 |   40.3      \n",
            "| 18    | 0.89388 |  0.85093 |   42.6      \n",
            "| 19    | 0.84482 |  0.89861 |   45.1      \n",
            "| 20    | 0.88616 |  0.90370 |   47.4      \n",
            "| 21    | 0.89542 |  0.86852 |   49.8      \n",
            "| 22    | 0.89106 |  0.86620 |   52.1      \n",
            "| 23    | 0.87532 |  0.84398 |   54.3      \n",
            "| 24    | 0.88970 |  0.85231 |   56.8      \n",
            "| 25    | 0.88099 |  0.87917 |   59.0      \n",
            "Training done in 59.026 seconds.\n",
            "---------------------------------------\n",
            "validation roc_auc model 1 fold- 4 :  0.9037037037037037\n",
            "Device used : cuda\n",
            "Will train until validation stopping metric hasn't improved in 100 rounds.\n",
            "---------------------------------------\n",
            "| EPOCH |  train  |   valid  | total time (s)\n",
            "| 1     | 0.50200 |  0.40139 |   2.5       \n",
            "| 2     | 0.52387 |  0.53426 |   4.8       \n",
            "| 3     | 0.63144 |  0.64167 |   7.1       \n",
            "| 4     | 0.69961 |  0.88056 |   9.4       \n",
            "| 5     | 0.74889 |  0.90324 |   11.7      \n",
            "| 6     | 0.74366 |  0.75787 |   14.0      \n",
            "| 7     | 0.77511 |  0.82083 |   16.2      \n",
            "| 8     | 0.77689 |  0.89306 |   18.5      \n",
            "| 9     | 0.76527 |  0.76296 |   20.8      \n",
            "| 10    | 0.73654 |  0.81435 |   22.9      \n",
            "| 11    | 0.74012 |  0.82824 |   25.2      \n",
            "| 12    | 0.75738 |  0.93519 |   27.6      \n",
            "| 13    | 0.77346 |  0.77407 |   30.2      \n",
            "| 14    | 0.80743 |  0.69028 |   32.5      \n",
            "| 15    | 0.84165 |  0.82037 |   34.7      \n",
            "| 16    | 0.86433 |  0.81759 |   37.0      \n",
            "| 17    | 0.87103 |  0.84676 |   39.3      \n",
            "| 18    | 0.86632 |  0.84537 |   41.5      \n",
            "| 19    | 0.87939 |  0.91806 |   43.9      \n",
            "| 20    | 0.88010 |  0.87593 |   46.1      \n",
            "| 21    | 0.88235 |  0.92130 |   48.5      \n",
            "| 22    | 0.86567 |  0.92176 |   50.8      \n",
            "| 23    | 0.88593 |  0.90648 |   53.0      \n",
            "| 24    | 0.91312 |  0.90648 |   55.3      \n",
            "| 25    | 0.92041 |  0.77685 |   57.5      \n",
            "Training done in 57.508 seconds.\n",
            "---------------------------------------\n",
            "validation roc_auc model 1 fold- 5 :  0.9351851851851851\n",
            "Device used : cuda\n",
            "Will train until validation stopping metric hasn't improved in 100 rounds.\n",
            "---------------------------------------\n",
            "| EPOCH |  train  |   valid  | total time (s)\n",
            "| 1     | 0.54467 |  0.52315 |   2.3       \n",
            "| 2     | 0.58565 |  0.73750 |   4.6       \n",
            "| 3     | 0.66007 |  0.59352 |   7.0       \n",
            "| 4     | 0.72151 |  0.76204 |   9.4       \n",
            "| 5     | 0.71329 |  0.77454 |   11.9      \n",
            "| 6     | 0.74820 |  0.74537 |   14.1      \n",
            "| 7     | 0.78746 |  0.90417 |   16.3      \n",
            "| 8     | 0.81717 |  0.94583 |   18.6      \n",
            "| 9     | 0.78783 |  0.94676 |   20.8      \n",
            "| 10    | 0.83247 |  0.89954 |   23.3      \n",
            "| 11    | 0.84699 |  0.92222 |   25.6      \n",
            "| 12    | 0.87925 |  0.94352 |   27.9      \n",
            "| 13    | 0.87888 |  0.93148 |   30.1      \n",
            "| 14    | 0.90363 |  0.94444 |   32.3      \n",
            "| 15    | 0.90521 |  0.93704 |   34.6      \n",
            "| 16    | 0.89180 |  0.94352 |   36.9      \n",
            "| 17    | 0.88239 |  0.94676 |   39.2      \n",
            "| 18    | 0.89927 |  0.84537 |   41.6      \n",
            "| 19    | 0.91569 |  0.95741 |   43.9      \n",
            "| 20    | 0.92802 |  0.92130 |   46.0      \n",
            "| 21    | 0.92140 |  0.87222 |   48.3      \n",
            "| 22    | 0.92707 |  0.86065 |   50.5      \n",
            "| 23    | 0.91649 |  0.84722 |   52.7      \n",
            "| 24    | 0.91384 |  0.83287 |   54.9      \n",
            "| 25    | 0.92238 |  0.72593 |   57.1      \n",
            "Training done in 57.118 seconds.\n",
            "---------------------------------------\n",
            "validation roc_auc model 1 fold- 6 :  0.9574074074074075\n",
            "Device used : cuda\n",
            "Will train until validation stopping metric hasn't improved in 100 rounds.\n",
            "---------------------------------------\n",
            "| EPOCH |  train  |   valid  | total time (s)\n",
            "| 1     | 0.47165 |  0.53148 |   2.4       \n",
            "| 2     | 0.54018 |  0.58611 |   4.8       \n",
            "| 3     | 0.60610 |  0.51574 |   7.1       \n",
            "| 4     | 0.77368 |  0.75139 |   9.4       \n",
            "| 5     | 0.74788 |  0.60741 |   11.7      \n",
            "| 6     | 0.82564 |  0.65741 |   14.0      \n",
            "| 7     | 0.80945 |  0.83380 |   16.5      \n",
            "| 8     | 0.79794 |  0.82361 |   18.8      \n",
            "| 9     | 0.86102 |  0.80833 |   21.1      \n",
            "| 10    | 0.84377 |  0.80000 |   23.3      \n",
            "| 11    | 0.82263 |  0.85741 |   25.8      \n",
            "| 12    | 0.83325 |  0.79722 |   28.1      \n",
            "| 13    | 0.86783 |  0.81759 |   30.3      \n",
            "| 14    | 0.89867 |  0.65556 |   32.4      \n",
            "| 15    | 0.89983 |  0.78333 |   34.6      \n",
            "| 16    | 0.89022 |  0.87546 |   37.0      \n",
            "| 17    | 0.89580 |  0.78333 |   39.3      \n",
            "| 18    | 0.88975 |  0.79167 |   41.5      \n",
            "| 19    | 0.91267 |  0.76713 |   43.6      \n",
            "| 20    | 0.89689 |  0.76898 |   45.8      \n",
            "| 21    | 0.90189 |  0.82917 |   48.2      \n",
            "| 22    | 0.92580 |  0.85139 |   50.5      \n",
            "| 23    | 0.93312 |  0.77315 |   52.9      \n",
            "| 24    | 0.93289 |  0.33889 |   55.1      \n",
            "| 25    | 0.94350 |  0.74306 |   57.4      \n",
            "Training done in 57.439 seconds.\n",
            "---------------------------------------\n",
            "validation roc_auc model 1 fold- 7 :  0.875462962962963\n",
            "Device used : cuda\n",
            "Will train until validation stopping metric hasn't improved in 100 rounds.\n",
            "---------------------------------------\n",
            "| EPOCH |  train  |   valid  | total time (s)\n",
            "| 1     | 0.52822 |  0.36852 |   2.2       \n",
            "| 2     | 0.56386 |  0.58148 |   4.6       \n",
            "| 3     | 0.66930 |  0.80463 |   6.9       \n",
            "| 4     | 0.77741 |  0.77361 |   9.1       \n",
            "| 5     | 0.77973 |  0.77269 |   11.3      \n",
            "| 6     | 0.79445 |  0.83472 |   13.7      \n",
            "| 7     | 0.82946 |  0.92870 |   16.0      \n",
            "| 8     | 0.84047 |  0.76852 |   18.2      \n",
            "| 9     | 0.79477 |  0.81806 |   20.4      \n",
            "| 10    | 0.81158 |  0.78472 |   22.7      \n",
            "| 11    | 0.84718 |  0.85509 |   25.2      \n",
            "| 12    | 0.84428 |  0.86019 |   27.7      \n",
            "| 13    | 0.84311 |  0.84167 |   29.9      \n",
            "| 14    | 0.88592 |  0.84167 |   32.2      \n",
            "| 15    | 0.89735 |  0.82917 |   34.6      \n",
            "| 16    | 0.89701 |  0.85741 |   36.8      \n",
            "| 17    | 0.87740 |  0.93611 |   39.1      \n",
            "| 18    | 0.90225 |  0.94306 |   41.3      \n",
            "| 19    | 0.88858 |  0.90556 |   43.9      \n",
            "| 20    | 0.87246 |  0.90741 |   46.2      \n",
            "| 21    | 0.89757 |  0.90741 |   48.4      \n",
            "| 22    | 0.90863 |  0.87546 |   50.5      \n",
            "| 23    | 0.91461 |  0.80556 |   52.8      \n",
            "| 24    | 0.91244 |  0.83981 |   55.0      \n",
            "| 25    | 0.91691 |  0.88426 |   57.4      \n",
            "Training done in 57.442 seconds.\n",
            "---------------------------------------\n",
            "validation roc_auc model 1 fold- 8 :  0.9430555555555556\n",
            "Device used : cuda\n",
            "Will train until validation stopping metric hasn't improved in 100 rounds.\n",
            "---------------------------------------\n",
            "| EPOCH |  train  |   valid  | total time (s)\n",
            "| 1     | 0.56669 |  0.41461 |   2.4       \n",
            "| 2     | 0.55353 |  0.79733 |   4.8       \n",
            "| 3     | 0.62733 |  0.70679 |   7.1       \n",
            "| 4     | 0.74034 |  0.75874 |   9.3       \n",
            "| 5     | 0.84185 |  0.91667 |   11.5      \n",
            "| 6     | 0.82204 |  0.88632 |   13.8      \n",
            "| 7     | 0.82678 |  0.86574 |   16.1      \n",
            "| 8     | 0.82001 |  0.90432 |   18.5      \n",
            "| 9     | 0.85163 |  0.89455 |   20.7      \n",
            "| 10    | 0.88681 |  0.81842 |   23.0      \n",
            "| 11    | 0.82617 |  0.89918 |   25.3      \n",
            "| 12    | 0.88311 |  0.92798 |   27.5      \n",
            "| 13    | 0.87203 |  0.78344 |   29.7      \n",
            "| 14    | 0.83482 |  0.78652 |   32.1      \n",
            "| 15    | 0.86653 |  0.85648 |   34.2      \n",
            "| 16    | 0.89552 |  0.86368 |   36.4      \n",
            "| 17    | 0.90690 |  0.85802 |   38.6      \n",
            "| 18    | 0.91303 |  0.85700 |   40.8      \n",
            "| 19    | 0.92500 |  0.78858 |   42.9      \n",
            "| 20    | 0.89420 |  0.71296 |   45.1      \n",
            "| 21    | 0.82075 |  0.71451 |   47.4      \n",
            "| 22    | 0.82392 |  0.77675 |   49.5      \n",
            "| 23    | 0.83527 |  0.82510 |   51.7      \n",
            "| 24    | 0.89089 |  0.85494 |   53.8      \n",
            "| 25    | 0.86966 |  0.89763 |   56.1      \n",
            "Training done in 56.087 seconds.\n",
            "---------------------------------------\n",
            "validation roc_auc model 1 fold- 9 :  0.9279835390946501\n",
            "Device used : cuda\n",
            "Will train until validation stopping metric hasn't improved in 100 rounds.\n",
            "---------------------------------------\n",
            "| EPOCH |  train  |   valid  | total time (s)\n",
            "| 1     | 0.51565 |  0.62037 |   2.4       \n",
            "| 2     | 0.56878 |  0.58025 |   4.5       \n",
            "| 3     | 0.61751 |  0.81173 |   6.8       \n",
            "| 4     | 0.73647 |  0.73354 |   9.1       \n",
            "| 5     | 0.68534 |  0.88169 |   11.4      \n",
            "| 6     | 0.81010 |  0.81121 |   13.7      \n",
            "| 7     | 0.82852 |  0.80967 |   16.0      \n",
            "| 8     | 0.85019 |  0.79424 |   18.2      \n",
            "| 9     | 0.81575 |  0.85854 |   20.4      \n",
            "| 10    | 0.85161 |  0.85751 |   22.6      \n",
            "| 11    | 0.87339 |  0.84362 |   24.7      \n",
            "| 12    | 0.88972 |  0.80504 |   26.9      \n",
            "| 13    | 0.90041 |  0.76235 |   29.2      \n",
            "| 14    | 0.82193 |  0.73508 |   31.7      \n",
            "| 15    | 0.80346 |  0.87140 |   33.8      \n",
            "| 16    | 0.81244 |  0.86265 |   36.0      \n",
            "| 17    | 0.87757 |  0.81533 |   38.1      \n",
            "| 18    | 0.88799 |  0.78909 |   40.4      \n",
            "| 19    | 0.87967 |  0.88580 |   42.8      \n",
            "| 20    | 0.86631 |  0.70267 |   45.0      \n",
            "| 21    | 0.77289 |  0.75412 |   47.5      \n",
            "| 22    | 0.83802 |  0.44856 |   49.8      \n",
            "| 23    | 0.83922 |  0.66924 |   52.0      \n",
            "| 24    | 0.76082 |  0.49537 |   54.2      \n",
            "| 25    | 0.64600 |  0.61626 |   56.4      \n",
            "Training done in 56.377 seconds.\n",
            "---------------------------------------\n",
            "validation roc_auc model 1 fold- 10 :  0.8858024691358025\n",
            "Device used : cuda\n",
            "Will train until validation stopping metric hasn't improved in 100 rounds.\n",
            "---------------------------------------\n",
            "| EPOCH |  train  |   valid  | total time (s)\n",
            "| 1     | 0.53160 |  0.61523 |   2.2       \n",
            "| 2     | 0.52915 |  0.63992 |   4.6       \n",
            "| 3     | 0.63251 |  0.67850 |   6.9       \n",
            "| 4     | 0.69295 |  0.70833 |   9.3       \n",
            "| 5     | 0.71991 |  0.83848 |   11.6      \n",
            "| 6     | 0.73659 |  0.73508 |   13.8      \n",
            "| 7     | 0.73485 |  0.82665 |   15.9      \n",
            "| 8     | 0.80821 |  0.86677 |   18.3      \n",
            "| 9     | 0.82526 |  0.79578 |   20.7      \n",
            "| 10    | 0.85830 |  0.79321 |   23.1      \n",
            "| 11    | 0.86911 |  0.75720 |   25.4      \n",
            "| 12    | 0.88551 |  0.77675 |   27.7      \n",
            "| 13    | 0.90293 |  0.86368 |   29.9      \n",
            "| 14    | 0.87856 |  0.68879 |   32.1      \n",
            "| 15    | 0.87061 |  0.76492 |   34.4      \n",
            "| 16    | 0.89980 |  0.88940 |   36.8      \n",
            "| 17    | 0.91597 |  0.84259 |   39.3      \n",
            "| 18    | 0.89652 |  0.70885 |   41.6      \n",
            "| 19    | 0.89593 |  0.68621 |   44.0      \n",
            "| 20    | 0.87879 |  0.85751 |   46.3      \n",
            "| 21    | 0.91078 |  0.81173 |   48.6      \n",
            "| 22    | 0.90310 |  0.71451 |   51.0      \n",
            "| 23    | 0.90440 |  0.45730 |   53.4      \n",
            "| 24    | 0.90591 |  0.75720 |   55.6      \n",
            "| 25    | 0.90221 |  0.71914 |   57.7      \n",
            "Training done in 57.737 seconds.\n",
            "---------------------------------------\n",
            "validation roc_auc model 1 fold- 11 :  0.88940329218107\n",
            "Device used : cuda\n",
            "Will train until validation stopping metric hasn't improved in 100 rounds.\n",
            "---------------------------------------\n",
            "| EPOCH |  train  |   valid  | total time (s)\n",
            "| 1     | 0.51895 |  0.59259 |   2.4       \n",
            "| 2     | 0.51452 |  0.42181 |   4.9       \n",
            "| 3     | 0.61648 |  0.62860 |   7.2       \n",
            "| 4     | 0.72847 |  0.67284 |   9.6       \n",
            "| 5     | 0.74764 |  0.68364 |   11.8      \n",
            "| 6     | 0.80032 |  0.47634 |   14.0      \n",
            "| 7     | 0.77181 |  0.71656 |   16.4      \n",
            "| 8     | 0.80573 |  0.80864 |   18.9      \n",
            "| 9     | 0.84616 |  0.83848 |   21.1      \n",
            "| 10    | 0.85957 |  0.83488 |   23.4      \n",
            "| 11    | 0.86206 |  0.77418 |   25.6      \n",
            "| 12    | 0.87175 |  0.85648 |   27.8      \n",
            "| 13    | 0.85272 |  0.80761 |   30.1      \n",
            "| 14    | 0.86866 |  0.61574 |   32.4      \n",
            "| 15    | 0.87649 |  0.72737 |   34.9      \n",
            "| 16    | 0.88115 |  0.82407 |   37.1      \n",
            "| 17    | 0.85851 |  0.85185 |   39.4      \n",
            "| 18    | 0.91311 |  0.88066 |   41.6      \n",
            "| 19    | 0.92197 |  0.85700 |   43.9      \n",
            "| 20    | 0.90277 |  0.82613 |   46.1      \n",
            "| 21    | 0.93327 |  0.83230 |   48.4      \n",
            "| 22    | 0.92734 |  0.83693 |   50.7      \n",
            "| 23    | 0.92911 |  0.83025 |   53.1      \n",
            "| 24    | 0.92902 |  0.85340 |   55.4      \n",
            "| 25    | 0.93868 |  0.82819 |   57.6      \n",
            "Training done in 57.561 seconds.\n",
            "---------------------------------------\n",
            "validation roc_auc model 1 fold- 12 :  0.8806584362139918\n",
            "Device used : cuda\n",
            "Will train until validation stopping metric hasn't improved in 100 rounds.\n",
            "---------------------------------------\n",
            "| EPOCH |  train  |   valid  | total time (s)\n",
            "| 1     | 0.55890 |  0.52469 |   2.4       \n",
            "| 2     | 0.53879 |  0.48971 |   4.6       \n",
            "| 3     | 0.65161 |  0.74383 |   6.9       \n",
            "| 4     | 0.74022 |  0.77263 |   9.1       \n",
            "| 5     | 0.76074 |  0.81481 |   11.4      \n",
            "| 6     | 0.78626 |  0.74640 |   13.6      \n",
            "| 7     | 0.83999 |  0.72634 |   15.9      \n",
            "| 8     | 0.84164 |  0.79784 |   18.1      \n",
            "| 9     | 0.79817 |  0.77418 |   20.4      \n",
            "| 10    | 0.84517 |  0.74177 |   22.7      \n",
            "| 11    | 0.87684 |  0.70679 |   24.9      \n",
            "| 12    | 0.87904 |  0.54012 |   27.0      \n",
            "| 13    | 0.85682 |  0.57665 |   29.2      \n",
            "| 14    | 0.88625 |  0.76337 |   31.4      \n",
            "| 15    | 0.88576 |  0.60340 |   33.6      \n",
            "| 16    | 0.87919 |  0.70370 |   35.8      \n",
            "| 17    | 0.86836 |  0.61780 |   38.0      \n",
            "| 18    | 0.90749 |  0.65381 |   40.3      \n",
            "| 19    | 0.92461 |  0.71553 |   42.5      \n",
            "| 20    | 0.92023 |  0.76183 |   44.7      \n",
            "| 21    | 0.91772 |  0.65484 |   46.9      \n",
            "| 22    | 0.91352 |  0.67387 |   49.3      \n",
            "| 23    | 0.93992 |  0.66049 |   51.5      \n",
            "| 24    | 0.94467 |  0.74743 |   53.7      \n",
            "| 25    | 0.94030 |  0.76595 |   55.9      \n",
            "Training done in 55.895 seconds.\n",
            "---------------------------------------\n",
            "validation roc_auc model 1 fold- 13 :  0.8148148148148148\n",
            "Device used : cuda\n",
            "Will train until validation stopping metric hasn't improved in 100 rounds.\n",
            "---------------------------------------\n",
            "| EPOCH |  train  |   valid  | total time (s)\n",
            "| 1     | 0.51241 |  0.54630 |   2.3       \n",
            "| 2     | 0.56012 |  0.60957 |   4.5       \n",
            "| 3     | 0.65445 |  0.65792 |   6.6       \n",
            "| 4     | 0.66476 |  0.82099 |   8.9       \n",
            "| 5     | 0.73057 |  0.87243 |   11.2      \n",
            "| 6     | 0.70893 |  0.76955 |   13.4      \n",
            "| 7     | 0.69290 |  0.79218 |   15.6      \n",
            "| 8     | 0.72520 |  0.79835 |   17.7      \n",
            "| 9     | 0.76337 |  0.88992 |   20.1      \n",
            "| 10    | 0.76564 |  0.74331 |   22.2      \n",
            "| 11    | 0.75560 |  0.81533 |   24.6      \n",
            "| 12    | 0.78237 |  0.82459 |   26.8      \n",
            "| 13    | 0.82159 |  0.83488 |   29.0      \n",
            "| 14    | 0.86870 |  0.93056 |   31.1      \n",
            "| 15    | 0.87630 |  0.95216 |   33.4      \n",
            "| 16    | 0.87327 |  0.92953 |   35.7      \n",
            "| 17    | 0.87549 |  0.95782 |   38.0      \n",
            "| 18    | 0.90771 |  0.95988 |   40.3      \n",
            "| 19    | 0.89739 |  0.94753 |   42.7      \n",
            "| 20    | 0.87931 |  0.96605 |   45.2      \n",
            "| 21    | 0.91465 |  0.94753 |   47.4      \n",
            "| 22    | 0.91191 |  0.97325 |   49.6      \n",
            "| 23    | 0.90033 |  0.98457 |   51.8      \n",
            "| 24    | 0.90259 |  0.96399 |   54.0      \n",
            "| 25    | 0.91917 |  0.90226 |   56.1      \n",
            "Training done in 56.080 seconds.\n",
            "---------------------------------------\n",
            "validation roc_auc model 1 fold- 14 :  0.9845679012345678\n",
            "Device used : cuda\n",
            "Will train until validation stopping metric hasn't improved in 100 rounds.\n",
            "---------------------------------------\n",
            "| EPOCH |  train  |   valid  | total time (s)\n",
            "| 1     | 0.53834 |  0.47325 |   2.2       \n",
            "| 2     | 0.57566 |  0.70679 |   4.4       \n",
            "| 3     | 0.58286 |  0.66872 |   6.8       \n",
            "| 4     | 0.69927 |  0.74023 |   9.1       \n",
            "| 5     | 0.70346 |  0.67335 |   11.2      \n",
            "| 6     | 0.73972 |  0.71142 |   13.3      \n",
            "| 7     | 0.79078 |  0.79012 |   15.5      \n",
            "| 8     | 0.84298 |  0.79733 |   17.7      \n",
            "| 9     | 0.85795 |  0.83745 |   19.9      \n",
            "| 10    | 0.87268 |  0.83693 |   22.3      \n",
            "| 11    | 0.88567 |  0.83745 |   24.5      \n",
            "| 12    | 0.87765 |  0.91718 |   26.9      \n",
            "| 13    | 0.87883 |  0.82767 |   29.1      \n",
            "| 14    | 0.89313 |  0.82305 |   31.7      \n",
            "| 15    | 0.90773 |  0.80041 |   34.0      \n",
            "| 16    | 0.90821 |  0.72016 |   36.4      \n",
            "| 17    | 0.90997 |  0.69290 |   38.6      \n",
            "| 18    | 0.80679 |  0.88169 |   40.8      \n",
            "| 19    | 0.80234 |  0.59722 |   43.2      \n",
            "| 20    | 0.80505 |  0.86523 |   45.5      \n",
            "| 21    | 0.83960 |  0.88169 |   47.8      \n",
            "| 22    | 0.88642 |  0.78395 |   50.0      \n",
            "| 23    | 0.90105 |  0.95062 |   52.2      \n",
            "| 24    | 0.89573 |  0.79835 |   54.4      \n",
            "| 25    | 0.89945 |  0.86934 |   56.7      \n",
            "Training done in 56.729 seconds.\n",
            "---------------------------------------\n",
            "validation roc_auc model 1 fold- 15 :  0.9506172839506173\n",
            "OOF ROC_AUC:-  0.8915328498661832\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fs949IwHFBjB",
        "colab_type": "text"
      },
      "source": [
        "# **Submission :**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F7mfK2_uQHqR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "submission_df       = pd.read_csv(\"/content/drive/My Drive/MachineHack/Anomalies_Detection/Sample_submission.csv\")\n",
        "submission_df['Class'] = y_pred_final"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pir3WRkARgRc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "submission_df.to_csv('tabnet.csv',index=False)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UoaRLNGXRrYo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "14b43e30-8831-4907-aaf6-f03ff65822ed"
      },
      "source": [
        "submission_df['Class'].mean()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.08726631907116354"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h2ainQpbsdYp",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "outputId": "3138ee24-296a-4412-c28e-f00855341f8c"
      },
      "source": [
        "submission_df.head()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Class</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.310625</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.038028</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.072966</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.319686</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.417845</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      Class\n",
              "0  0.310625\n",
              "1  0.038028\n",
              "2  0.072966\n",
              "3  0.319686\n",
              "4  0.417845"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    }
  ]
}